In [96]:
!pip install git+http://git@github.com/kavgan/word_cloud.git
In [92]:
In [1]:
!pip install wordcloud
In [2]:
!pip install gensim
In [4]:
!pip install stop_words
In [5]:
import numpy as np
import io
import pandas as pd
from stop_words import get_stop_words
from nltk.tokenize import RegexpTokenizer
from gensim import corpora, models
import gensim
import matplotlib.pyplot as plt
from wordcloud import WordCloud
from PIL import Image
import PIL.ImageOps
import random
from wordcloud import ImageColorGenerator
%matplotlib inline
%config InlineBackend.figure_format='retina'
In [10]:
df = pd.read_csv('small_700_through_710_descr_clm_code.csv')
df.drop('Unnamed: 0',axis=1, inplace=True)
df['descr_clm'] = df.descr + df.clm
df.drop(['descr','clm'],axis=1, inplace=True)
df['code'] = df['code'].astype('category')
df.head()
Out[10]:
Word Cloud for 705¶
In [11]:
df_705 = df[df['code']==705]
In [12]:
df_705.head()
Out[12]:
In [73]:
from wordcloud import STOPWORDS
In [83]:
custom_stopword_list = ['using','said','based','wherein','comprising','comprise','diagram','system','process','method','one', 'may','claim','embodiment','invention','include', 'example', 'include','step','figure','fig']
In [84]:
stopwords = STOPWORDS
stopwords |= set(custom_stopword_list)
In [85]:
text = df_705.descr_clm.values
wordcloud = WordCloud(
width = 3000,
height = 2000,
background_color = 'black',
stopwords = stopwords).generate(str(text))
fig = plt.figure(
figsize = (40, 30),
facecolor = 'k',
edgecolor = 'k')
plt.imshow(wordcloud, interpolation = 'bilinear')
plt.axis('off')
plt.tight_layout(pad=0)
plt.show()
Word Cloud for 706¶
In [80]:
df_706 = df[df['code']==706]
In [81]:
df_706.head()
Out[81]:
In [106]:
text_706 = df_706.descr_clm.values
wordcloud = WordCloud(
width = 3000,
height = 2000,
background_color = 'black',
stopwords = stopwords).generate(str(text_706))
fig = plt.figure(
figsize = (40, 30),
facecolor = 'k',
edgecolor = 'k')
plt.imshow(wordcloud, interpolation = 'bilinear')
plt.axis('off')
plt.tight_layout(pad=0)
plt.show()
In [100]:
from word_cloud.word_cloud_generator import WordCloud
from IPython.core.display import HTML
wc=WordCloud(use_tfidf=True,stopwords=stopwords)
text = sort(text)
#don't randomize color, show only top 500
embed_code=wc.get_embed_code(text=text,random_color=True,topn=500)
HTML(embed_code)
topic modeling¶
In [108]:
!pip install pyLDAvis
In [109]:
from __future__ import print_function
import pyLDAvis
import pyLDAvis.sklearn
pyLDAvis.enable_notebook()
In [110]:
from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import LatentDirichletAllocation
In [135]:
df_705_706 = df[(df['code'] == 705)|(df['code'] == 706)]
In [136]:
df_705_706['code'].unique()
Out[136]:
In [144]:
df_705_706['code'].value_counts()
Out[144]:
In [160]:
tf_vectorizer_705_706 = CountVectorizer(strip_accents = 'unicode',
stop_words = stopwords,
lowercase = True,
token_pattern = r'\b[a-zA-Z]{3,}\b',
max_df = 0.5,
min_df = 10)
dtm_tf_705_706 = tf_vectorizer_705_706.fit_transform(df_705_706.descr_clm.values)
print(dtm_tf_705_706.shape)
In [161]:
tfidf_vectorizer_705_706 = TfidfVectorizer(**tf_vectorizer.get_params())
dtm_tfidf_705_706 = tfidf_vectorizer_705_706.fit_transform(df_705_706.descr_clm.values)
print(dtm_tfidf_705_706.shape)
In [162]:
# for TF DTM
lda_tf_705_706 = LatentDirichletAllocation(n_topics=2, random_state=0)
lda_tf_705_706.fit(dtm_tf_705_706)
# for TFIDF DTM
lda_tfidf_705_706 = LatentDirichletAllocation(n_topics=2, random_state=0)
lda_tfidf_705_706.fit(dtm_tfidf_705_706)
Out[162]:
In [163]:
pyLDAvis.sklearn.prepare(lda_tf_705_706, dtm_tf_705_706, tf_vectorizer_705_706)
Out[163]:
In [164]:
pyLDAvis.sklearn.prepare(lda_tfidf_705_706, dtm_tfidf_705_706, tfidf_vectorizer_705_706)
Out[164]:
Topic Modeling 705¶
In [165]:
tf_vectorizer_705 = CountVectorizer(strip_accents = 'unicode',
stop_words = stopwords,
lowercase = True,
token_pattern = r'\b[a-zA-Z]{3,}\b',
max_df = 0.5,
min_df = 10)
dtm_tf_705 = tf_vectorizer_705.fit_transform(df_705.descr_clm.values)
print(dtm_tf_705.shape)
In [166]:
tfidf_vectorizer_705 = TfidfVectorizer(**tf_vectorizer.get_params())
dtm_tfidf_705 = tfidf_vectorizer_705.fit_transform(df_705.descr_clm.values)
print(dtm_tfidf_705.shape)
In [171]:
# for TF DTM
lda_tf_705 = LatentDirichletAllocation(n_topics=5, random_state=0)
lda_tf.fit(dtm_tf_705)
# for TFIDF DTM
lda_tfidf_705 = LatentDirichletAllocation(n_topics=5, random_state=0)
lda_tfidf_705.fit(dtm_tfidf_705)
Out[171]:
In [172]:
pyLDAvis.sklearn.prepare(lda_tf_705, dtm_tf_705, tf_vectorizer_705)
In [173]:
pyLDAvis.sklearn.prepare(lda_tfidf_705, dtm_tfidf_705, tfidf_vectorizer_705)
Out[173]:
Topic Modelling 706¶
In [174]:
tf_vectorizer_706 = CountVectorizer(strip_accents = 'unicode',
stop_words = stopwords,
lowercase = True,
token_pattern = r'\b[a-zA-Z]{3,}\b',
max_df = 0.5,
min_df = 10)
dtm_tf_706 = tf_vectorizer_706.fit_transform(df_706.descr_clm.values)
print(dtm_tf_706.shape)
In [175]:
tfidf_vectorizer_706 = TfidfVectorizer(**tf_vectorizer.get_params())
dtm_tfidf_706 = tfidf_vectorizer_706.fit_transform(df_706.descr_clm.values)
print(dtm_tfidf_706.shape)
In [176]:
# for TF DTM
lda_tf_706 = LatentDirichletAllocation(n_topics=5, random_state=0)
lda_tf_706.fit(dtm_tf_706)
# for TFIDF DTM
lda_tfidf_706 = LatentDirichletAllocation(n_topics=5, random_state=0)
lda_tfidf_706.fit(dtm_tfidf_706)
Out[176]:
In [177]:
pyLDAvis.sklearn.prepare(lda_tf_706, dtm_tf_706, tf_vectorizer_706)
Out[177]:
In [178]:
pyLDAvis.sklearn.prepare(lda_tfidf_706, dtm_tfidf_706, tfidf_vectorizer_706)
Out[178]:
In [ ]: